import time
import requests as r
from lxml import etree
import re
import psycopg2
from concurrent.futures import ThreadPoolExecutor

# 连接数据库
conn_pg = psycopg2.connect("host=localhost dbname=postgres user=postgres password=admin port=5432")
# 创建一个游标
cur = conn_pg.cursor()

def fetch_course_data(page):
    print(f"正在爬取{page}页")
    response = r.get(f"https://hebust.fanya.chaoxing.com/portal/courseNetwork/index?page={page}&keyword=")
    html = etree.HTML(response.text.replace(u'\xa0 ', u' '))
    result = html.xpath('/html/body/div/div[1]/div[2]/div//dd//a/text()')
    courseIdUrl = html.xpath('/html/body/div/div[1]/div[2]/div//dt//a/@href')
    result2 = html.xpath('/html/body/div/div[1]/div[2]/div/dl/dd[2]/text()')

    if len(result)!=len(courseIdUrl)!=result2:
        print(f"第{page}页配置有误")

    for elementIndex in range(len(result)):
        courseName = result[elementIndex]
        author = result2[elementIndex]
        courseId = re.search(r'/(\d+)', courseIdUrl[elementIndex]).group(1)
        cur.execute(
            f"INSERT INTO public.course(course_id, teacher_name, course_name)VALUES ({courseId},'{author}','{courseName}')")
    conn_pg.commit()
    time.sleep(0.5)

# 使用多线程
with ThreadPoolExecutor(max_workers=5) as executor:  # 你可以根据需要调整 max_workers
    pages = range(1, 619)
    executor.map(fetch_course_data, pages)

# 关闭游标
cur.close()
# 关闭连接
conn_pg.close()
